package com.nlp.visualization.crawler.processor;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;

import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * @author LXM
 * @Title: 新闻爬虫的抽象类
 * @Description: 不同的页面需要使用不同的爬虫处理程序，因此让它的子类来完成不同的爬取任务
 * @date 2018/3/3下午9:17
 */
public abstract class Processor {

    //日志
    protected Logger logger = LoggerFactory.getLogger(Processor.class);
    //日期
    protected Date taskDate;
    //统一Process的时间格式
    protected static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");

    public abstract void processor(Page page);

    public abstract void processor(Page page, String date);

    public abstract void init(Spider spider);

    protected String getSourceFromPage(Page page) {
        String type = page.getUrl().regex("http\\w?://\\S+\\.(\\S+)\\.com").toString();
        if (type != null && !"".equals(type)) {
            type = type.substring(0, 1).toUpperCase() + type.toLowerCase().substring(1);
        }
        return type;
    }

    protected void initAddRequest(Request request, Spider spider) {
        request.setPriority(1);
        spider.addRequest(request);
    }

    public void initProcessor(Spider spider) {
        this.taskDate = new Date();
        init(spider);
    }

    public void initProcessor(Spider spider, Date taskDate) {
        this.taskDate = taskDate;
        init(spider);
    }


}
